<?php 
namespace App\Http\Controllers\TongGaoApi;

use Illuminate\Http\Request;
use App\Http\Controllers\Controller;
use App\Models\Mysql\tonggao\article;
use App\Models\Mysql\tonggao\term;
use App\Models\Mysql\tonggao\term_detail;
use App\Models\Mysql\tonggao\sentence;
use Symfony\Component\DomCrawler\Crawler;
use App\libraries\diff\Diff;	

class NLPController extends Controller
{
//	use GlobalData;

	const ROUTE_PREFIX = '/api/*';

	protected $request;

	protected $curTime;

	protected $debug;


	/**
	 * 创建新的`DictController`控制器实例
	 * @param \Illuminate\Http\Request $request
	 */
	public function __construct(Request $request)
	{
		$this->request = $request;
		$this->error = [];
		$this->curTime = time();

		if(config('app.debug')) {
			$this->debug = true;
			error_reporting(E_ALL | E_STRICT);
		} else {
			$this->debug = false;
		}
	}
	
	public function searchHighFreq()
    {
		set_time_limit(0);
		$debug1 = true;
		$debug2 = false;
		$cont = true;
		
		$puncturesEN = array(",","\.","\?","!",":",'\"',"\'","-",";","\(","\)","\[","\]"," ");
		$puncturesCN = array("，","。","？","！","：",'“','”',"‘","’","-","；","《","》","、","（","）","…","·","—","　");
		$puncturesPattern="/[0-9]|[A-Z]|[a-z]|".implode("|",array_merge($puncturesEN,$puncturesCN))."/";	
		//echo "$pattern</br>";
		for($i=8;$i<=8;$i++){
			$temp_high_freq_table_name="temp_high_freq_$i";
			
			//term::select("DROP TABLE IF EXISTS `$temp_high_freq_table_name`");
			//term::select("CREATE TABLE `$temp_high_freq_table_name` (
			//	`pattern` varchar(180) NOT NULL,
			//	`allfreq` int(6) NOT NULL,
			//	`freq` int(6) NOT NULL
			//	) ENGINE=MyISAM DEFAULT CHARSET=utf8");
			//term::select("ALTER TABLE `$temp_high_freq_table_name`
			//	ADD KEY `pattern` (`pattern`)");
			/*
			term::select("CREATE TABLE `$temp_high_freq_table_name` (
				`bookname` varchar(120) NOT NULL,
				`pattern` varchar(180) NOT NULL,
				`allfreq` int(6) NOT NULL,
				`freq` int(6) NOT NULL
				) ENGINE=InnoDB DEFAULT CHARSET=utf8");
			term::select("ALTER TABLE `$temp_high_freq_table_name`
				ADD KEY `bookname` (`bookname`),
				ADD KEY `pattern` (`pattern`)");
			*/
		}
		
		for($i=8;$i<=8;$i++){
			$temp_high_freq_table_name="temp_high_freq_$i";
			$sentences = sentence::select("select sentence.content from sentence, article where sentence.article_id=article.id and substring(article.name,3,1)=\"$i\"");
			$subject = "";
			foreach($sentences as $sentence){
				$subject .= preg_replace($puncturesPattern,"z",$sentence->content);
			}
			if($cont){
				echo mb_strpos($subject,"就会慢慢建立起佛法的",98134);
			}
			/*for($j=0;$j<mb_strlen($subject);$j++){
				if (strlen(mb_substr($subject,$j,1))<>3)
					echo "Error ".mb_substr($subject,$j,1)." is not Chinese</br>";
			}*/
			//先试探
			$maxLen=(($i==7)?21+1:25+1);
			/*
			$maxLen=2;
			while(true){
				if ($debug1) {echo "start to search $maxLen</br>";ob_flush();flush();}
				$foundCount=1;
				for($j=0;$j<mb_strlen($subject)-$maxLen+1;$j++){
					$pattern=mb_substr($subject,$j,$maxLen);
					if (mb_strpos($pattern,"z")!==false)
						continue;
					if (mb_strpos($subject,$pattern,$j+1)!==false){
						$foundCount++;
						if ($debug2){
							echo "$pattern found maxLen=$maxLen</br>";
							ob_flush();flush();
						}
						break;
					}
				}
				if($foundCount==1)
					break;
				$maxLen++;
			}
			echo sprintf("final maxLen is %d </br>",$maxLen-1);
			ob_flush();flush();
			*/
			$outputFileName = iconv('UTF-8','GB2312',"F:\\000_booktran\\高频词\\book$i.txt");
			file_put_contents($outputFileName,"");
			//for($len=$maxLen-1;$len>=2;$len--){
			for($len=2;$len>=2;$len--){				
				if ($debug1) {echo "start to search $len</br>";ob_flush();flush();}
				//for($j=0;$j<mb_strlen($subject)-$len+1;$j++){
				for($j=98134;$j<mb_strlen($subject)-$len+1;$j++){	
					$allFoundCount=$foundCount=0;
					$pattern=mb_substr($subject,$j,$len);
					if (mb_strpos($pattern,"z")!==false)
						continue;
					//$temp_r = term::select("select * from $temp_high_freq_table_name where pattern like \"%$pattern%\" and bookname=\"wj$i\"");
					$temp_r = term::select("select * from $temp_high_freq_table_name where pattern like \"%$pattern%\"");
					$bigPatterns = array();
					foreach($temp_r as $temp_r_v){
						$offset = mb_strpos($temp_r_v->pattern,$pattern);
						$length = mb_strlen($temp_r_v->pattern);
						$bigPatterns []= array(
							'offset'=>$offset,
							'length'=>$length,
							'string'=>$temp_r_v->pattern
						);
					}
					
					$subjectLen = mb_strlen($subject);
					for($startPos=$j;$startPos<$subjectLen-$len+1;$startPos++){
						$foundPos=mb_strpos($subject,$pattern,$startPos);
						if ($foundPos!==false){
							$bigPatternHere = false;
							foreach($bigPatterns as $bigPattern){
								$bigPatternStartPos = $foundPos - $bigPattern['offset'];
								$bigPatternLength = $bigPattern['length']; 
								$bigPatternString = $bigPattern['string']; 
								$bigPatternEndPos = $bigPatternStartPos + $bigPatternLength - 1;
								if (($bigPatternStartPos<0)||($bigPatternEndPos>=$subjectLen))
									continue;
								if (mb_substr($subject,$bigPatternStartPos,$bigPatternLength)== $bigPatternString){
									$bigPatternHere = true;
									break;
								}
							}
							if (!$bigPatternHere){
								$foundCount++;
								if ($debug2) {echo "$pattern found $foundCount times</br>";ob_flush();flush();}
							}else{
								if ($debug2) {echo "$pattern in $bigPatternString</br>";ob_flush();flush();}
							}
							$startPos = $foundPos+1;
							$allFoundCount++;
						}else{
							break;
						}
					}
					if($allFoundCount>1){
						//term::select("insert into `$temp_high_freq_table_name` (bookname,pattern,allfreq,freq) VALUES (\"wj$i\",\"$pattern\",$allFoundCount,$foundCount)");
						term::select("insert into `$temp_high_freq_table_name` (pattern,allfreq,freq) VALUES (\"$pattern\",$allFoundCount,$foundCount)");						
						//file_put_contents($outputFileName,"wj$i,$pattern,$allFoundCount,$foundCount\r\n",8);
						if ($debug1) {echo "$i,$pattern,$allFoundCount,$foundCount</br>";ob_flush();flush();}
					}
				}
			}
		}
	}

	//TODO: use a db so lookup existing will be quicker
	public function searchHighFreq_v2()
    {
		set_time_limit(0);
		$termTableName ="term_78";
		
		$maxLen = 12;
		
		$puncturesEN = array(",",".","?","!",":",'"',"'","-",";");
		$puncturesCN = array("，","。","？","！","：",'“','”',"‘","’","-","；","《","》","、","（","）","……","…","·");
		/* 中英对照广论
		$fileName = iconv('UTF-8','GB2312',"F:\\00_NLP\\MyTB\\广论\\3-广论中英对照-全.txt");
		$file = file_get_contents($fileName);
		$inputLines = explode("\r\n",$file);*/
		
		//七八册引文
		$temp_r=sentence::select("select sentence_78.content from sentence_78, article_78 where article_78.id=sentence_78.article_id and article_type=1");
		$inputLines = array();
		foreach($temp_r as $v){
			$inputLines []= $v->content;
		}
		
		term::select("delete from $termTableName");
		//preprocessing
		$parts = array();
		foreach($inputLines as $k=>$line){
			if (empty($line))
				continue;
			
			//strip off 《》
			$startPos = 0;
			$bookNameFound=false;
			/*if (mb_strpos($line,"《",$startPos)!==false){
				echo "line=$line</br>";
				$bookNameFound=true;
			}*/
			while($startPos<=mb_strlen($line)-1){
				$startPos = mb_strpos($line,"《",$startPos);
				if ($startPos===FALSE)
					break;
				$endPos = mb_strpos($line,"》",$startPos+1);
				if ($endPos===FALSE){
					echo "Error</br>";
					//special case
					$endPos = $startPos+5;//mb_strpos($line,")",$startPos+1);
				}
				$bookName = mb_substr($line,$startPos+1,$endPos-$startPos-1);
				$temp_r = term::select("select * from $termTableName where chinese_key=\"$bookName\"");
				if (empty($temp_r)){
					term::select("insert into $termTableName (chinese_key,category) values (\"$bookName\",\"bookname\")");
				}
				
				//echo "bookName=$bookName startPos=$startPos endPos=$endPos</br>";
				$left = mb_substr($line,0,$startPos);
				$right = mb_substr($line,$endPos+1);
				$line = $left.$right;
				//echo "left=$left</br>right=$right</br>line=$line</br>";
				$startPos = $endPos + 1;
			}				
				
			$curLineLang = "CN";
			$punctures = ($curLineLang=="EN") ? $puncturesEN : $puncturesCN;
			//var_dump($punctures);
			
			foreach($punctures as $puncture)
				$line=str_replace($puncture," ",$line);
			$fields = explode(" ", $line);
			foreach($fields as $field){
				$parts []= $field;
			}
		}
		//found freq>=2
		$curLineLang = "CN";
		$partCount=count($parts);
		foreach($parts as $part_k=>$part){
			echo "part=$part</br>";
			$partLen=mb_strlen($part);
			for($unitLen=2;$unitLen<=$maxLen;$unitLen++){
				for($startPos=0;$startPos<$partLen-$unitLen+1;$startPos++){
					$curUnit = mb_substr($part,$startPos,$unitLen);
					
					//look up in existing first
					$temp_r = term::select("select id from $termTableName where chinese_key=\"$curUnit\"");
					if (!empty($temp_r))
						continue;
						
					$curUnitFreq = 1;
					echo "$curUnit</br>";
					for($part_k2=$part_k+1;$part_k2<$partCount;$part_k2++){
						$startPos2 = ($part_k2==$part_k) ? $startPos: 0;
						$partLen2 = mb_strlen($parts[$part_k2]);
						while($startPos2<$partLen2-1){
							$startPos2=mb_strpos($parts[$part_k2],$curUnit,$startPos2+1);
							if ($startPos2!==FALSE)
								$curUnitFreq++;
							else
								break;
						};
					}
					
					if($curUnitFreq>=2){
						term::select("insert into $termTableName (chinese_key,freq) values (\"$curUnit\",$curUnitFreq)");
						file_put_contents("E:\\HighFreq_GT2.txt",$curUnit."|".$curUnitFreq."\r\n",8);
					}
				}
			}
		}
	}

    //workable but use array
	public function searchHighFreq_v1()
    {
		set_time_limit(0);
		$HighFreqCount=1000;
		$HighFreqCountOne=1000;
		$HighFreqCN = $HighFreqEN=array();
		$maxLen = 12;
		for($i=1;$i<=$maxLen;$i++){
			$curHighFreqCount = ($i==1)?$HighFreqCountOne:$HighFreqCount;
			for($j=0;$j<$HighFreqCount;$j++){
				$HighFreq["CN"][$i][$j]['key'] = $HighFreq["EN"][$i][$j]['key'] = "";
				$HighFreq["CN"][$i][$j]['freq'] = $HighFreq["EN"][$i][$j]['freq'] = 0;
			}
		}
		
		$puncturesEN = array(",",".","?","!",":",'"',"'","-",";");
		$puncturesCN = array("，","。","？","！","：",'“','”',"‘","’","-","；","《","》","、","（","）","……","…","·");
		/* 中英对照广论
		$fileName = iconv('UTF-8','GB2312',"F:\\00_NLP\\MyTB\\广论\\3-广论中英对照-全.txt");
		$file = file_get_contents($fileName);
		$inputLines = explode("\r\n",$file);*/
		
		//七八册引文

		$temp_r=sentence::select("select sentence_78.content from sentence_78, article_78 where article_78.id=sentence_78.article_id and article_type=1");
		$inputLines = array();
		foreach($temp_r as $v){
			$inputLines []= $v->content;
		}
		
		//preprocessing
		$parts = array();
		foreach($inputLines as $k=>$line){
			if (empty($line))
				continue;
			$curLineLang = ($this::isEnglish($line))?"EN":"CN";
			
			$punctures = ($curLineLang=="EN") ? $puncturesEN : $puncturesCN;
			//var_dump($punctures);
			
			foreach($punctures as $puncture)
				$line=str_replace($puncture," ",$line);
			$fields = explode(" ", $line);
			foreach($fields as $field){
				$parts []= $field;
			}
		}
		
		//found freq>=2
		$curLineLang = "CN";
		$partCount=count($parts);
		foreach($parts as $part_k=>$part){
			echo "part=$part</br>";
			$partLen=mb_strlen($part);
			for($unitLen=2;$unitLen<=$maxLen;$unitLen++){
				$curHighFreqCount = ($unitLen==1)?$HighFreqCountOne:$HighFreqCount;
				//echo "curHighFreqCount=$curHighFreqCount</br>";
				for($startPos=0;$startPos<$partLen-$unitLen+1;$startPos++){
					$curUnit = mb_substr($part,$startPos,$unitLen);
					
					//look up in existing first
					$found = false;
					for($j=0;$j<$HighFreqCount;$j++){
						if (empty($HighFreq[$curLineLang][$unitLen][$j]['key'])){
							echo " count of $unitLen is $j</br>";
							break;
						}
						if ($curUnit==$HighFreq[$curLineLang][$unitLen][$j]['key']){
							$found=true;
							break;
						}
					}
					if($found)
						continue;
						
					$curUnitFreq = 1;
					echo "$curUnit</br>";
					for($part_k2=$part_k+1;$part_k2<$partCount;$part_k2++){
						//need to improve to include multiple occurence
						$startPos2 = ($part_k2==$part_k) ? $startPos: 0;
						$partLen2 = mb_strlen($parts[$part_k2]);
						while($startPos2<$partLen2-1){
							//echo "startPos=$startPos2</br>";
							//echo "partLen=$partLen</br>";
							//echo "curUnit=$curUnit</br>";
							//echo "{$parts[$part_k2]}</br>";
							$startPos2=mb_strpos($parts[$part_k2],$curUnit,$startPos2+1);
							if ($startPos2!==FALSE)
								$curUnitFreq++;
							else
								break;
						};
					}
					
					if($curUnitFreq>=2){
						
						$HighFreq[$curLineLang][$unitLen][$j]['key'] = $curUnit;
						$HighFreq[$curLineLang][$unitLen][$j]['freq'] = $curUnitFreq;
						file_put_contents("E:\\HighFreq_GT2.txt",$curUnit."|".$curUnitFreq."\r\n",8);
					}
				}
			}
		}
		//var_dump($inputLines);
		/* workable
		foreach($inputLines as $k=>$line){
			if (empty($line))
				continue;
			$curLineLang = ($this::isEnglish($line))?"EN":"CN";
			
			$punctures = ($curLineLang=="EN") ? $puncturesEN : $puncturesCN;
			var_dump($punctures);
			
			foreach($punctures as $puncture)
				$line=str_replace($puncture," ",$line);
			$fields = explode(" ", $line);
			
			foreach($fields as $field){
				$fieldLen=mb_strlen($field);
				for($unitLen=1;$unitLen<=$maxLen;$unitLen++){
					$curHighFreqCount = ($unitLen==1)?$HighFreqCountOne:$HighFreqCount;
					echo "curHighFreqCount=$curHighFreqCount</br>";
					for($startPos=0;$startPos<$fieldLen-$unitLen+1;$startPos++){
						$curUnit = mb_substr($field,$startPos,$unitLen);
						echo "$curUnit</br>";
						$found=false;
						for($j=0;$j<$HighFreqCount;$j++){
							if (empty($HighFreq[$curLineLang][$unitLen][$j]['key'])){
								echo "empty</br>";
								break;
							}
							if ($curUnit==$HighFreq[$curLineLang][$unitLen][$j]['key']){
								$found=true;
								break;
							}
						}
						if ($found){
							$HighFreq[$curLineLang][$unitLen][$j]['freq']++;
							$newFreq = $HighFreq[$curLineLang][$unitLen][$j]['freq'];
							
							//insIndex is the index of the last one with freq >=newFreq
							//insIndex=-1 means all freq are smaller than newFreq
							for($insIndex=$j-1;$insIndex>=0;$insIndex--){
								if($HighFreq[$curLineLang][$unitLen][$insIndex]['freq']>=$newFreq)
									break;
							}
							
							//curUnit will take insIndex+1
							//insIndex+1...j-1 will be moved to insIndex+2...j
							for($moveIndex=min($j,$curHighFreqCount-1);$moveIndex>=$insIndex+2;$moveIndex--){
								$HighFreq[$curLineLang][$unitLen][$moveIndex]['freq']=$HighFreq[$curLineLang][$unitLen][$moveIndex-1]['freq'];
								$HighFreq[$curLineLang][$unitLen][$moveIndex]['key']=$HighFreq[$curLineLang][$unitLen][$moveIndex-1]['key'];
							}
							
							$HighFreq[$curLineLang][$unitLen][$insIndex+1]['freq']=$newFreq;
							$HighFreq[$curLineLang][$unitLen][$insIndex+1]['key']=$curUnit;
														
						} else{
							if($j<$curHighFreqCount-1){
								echo "Going to add j=$j</br>";
								$HighFreq[$curLineLang][$unitLen][$j]['freq']=1;
								$HighFreq[$curLineLang][$unitLen][$j]['key']=$curUnit;
								//var_dump($HighFreq[$curLineLang][$unitLen]);
							}
						}
					}
				}
			}

		}
		
		for($i=1;$i<=$maxLen;$i++){
			$curHighFreqCount = ($i==1)?$HighFreqCountOne:$HighFreqCount;
			for($j=0;$j<$HighFreqCount;$j++){
				if (empty($HighFreq["CN"][$i][$j]['key']))
					break;
				file_put_contents("E:\\HighFreq_$i.txt",$HighFreq["CN"][$i][$j]['key']."|".$HighFreq["CN"][$i][$j]['freq']."\r\n",8);
			}
		}end of workable*/
	}
	
	public function removeChinese(){
		set_time_limit(0);
		$fileNameArray=array(
			//iconv('UTF-8','GB2312',"F:\\000_booktran\\combine5\\合并\\02-03\\英文比较03-2\\03-2-A-合稿.txt"),
			//iconv('UTF-8','GB2312',"F:\\000_booktran\\combine5\\合并\\02-03\\英文比较03-2\\03-2-B.txt"),
			//iconv('UTF-8','GB2312',"F:\\000_booktran\\combine5\\合并\\02-03\\英文比较02-1\\02-1-A-new-CaoYuanMei-LiShuAccepted.txt"),
			//iconv('UTF-8','GB2312',"F:\\000_booktran\\combine5\\合并\\02-03\\英文比较02-1\\02-1-B.txt"),
			//iconv('UTF-8','GB2312',"F:\\000_booktran\\combine5\\合并\\02-03\\英文比较02-2\\02-2-A-合稿初稿.txt"),
			//iconv('UTF-8','GB2312',"F:\\000_booktran\\combine5\\合并\\02-03\\英文比较02-2\\02-2-B.txt"),
			//iconv('UTF-8','GB2312',"F:\\000_booktran\\combine6\\初次合并\\02英文比较\\02-A合稿.txt"),
			//iconv('UTF-8','GB2312',"F:\\000_booktran\\combine6\\初次合并\\02英文比较\\02-B.txt"),
			iconv('UTF-8','GB2312',"F:\\000_booktran\\combine6\\初次合并\\03英文比较\\03-A合稿（校对完成）.txt"),
			iconv('UTF-8','GB2312',"F:\\000_booktran\\combine6\\初次合并\\03英文比较\\03-B.txt"),
			iconv('UTF-8','GB2312',"F:\\000_booktran\\combine6\\初次合并\\04英文比较\\04-A合稿（校对完成).txt"),
			iconv('UTF-8','GB2312',"F:\\000_booktran\\combine6\\初次合并\\04英文比较\\04-B.txt"),
			iconv('UTF-8','GB2312',"F:\\000_booktran\\combine6\\初次合并\\05英文比较\\05-A合稿（校对完成）.txt"),
			iconv('UTF-8','GB2312',"F:\\000_booktran\\combine6\\初次合并\\05英文比较\\05-B-v2.txt"),
		);
		foreach($fileNameArray as $fileName){
			$inputFile = file_get_contents($fileName);
			$outputFileName = str_replace(".txt","",$fileName)."-output.txt";
			file_put_contents($outputFileName,"");
			$inputLen = mb_strlen($inputFile);
			for($i=0;$i<$inputLen;$i++){
				$curChar = mb_substr($inputFile,$i,1);
				$curCharLen = strlen($curChar);
				echo "$curChar $curCharLen</br>";
				if ($curCharLen==1)
					file_put_contents($outputFileName,$curChar,8);
				else
					continue;
				/*
				if (($curChar=="\r") || ($curChar=="\n")){
					file_put_contents($outputFileName,8);
					continue;
				}
				*/
			}
		}
	}

	public function removeEnglish(){
		set_time_limit(0);
		$fileNameArray=array(
			//iconv('UTF-8','GB2312',"F:\\000_booktran\\combine5\\合并\\02-03\\英文比较03-2\\03-2-A-合稿.txt"),
			//iconv('UTF-8','GB2312',"F:\\000_booktran\\combine5\\合并\\02-03\\英文比较03-2\\03-2-B.txt"),
			//iconv('UTF-8','GB2312',"F:\\000_booktran\\combine5\\合并\\02-03\\英文比较02-1\\02-1-A-new-CaoYuanMei-LiShuAccepted.txt"),
			//iconv('UTF-8','GB2312',"F:\\000_booktran\\combine5\\合并\\02-03\\英文比较02-1\\02-1-B.txt"),
			//iconv('UTF-8','GB2312',"F:\\000_booktran\\combine5\\合并\\02-03\\英文比较02-2\\02-2-A-合稿初稿.txt"),
			//iconv('UTF-8','GB2312',"F:\\000_booktran\\combine5\\合并\\02-03\\英文比较02-2\\02-2-B.txt"),
			iconv('UTF-8','GB2312',"F:\\000_booktran\\combine5\\合并\\01-1\\术语替换\\返稿\\原文比较\\A.txt"),
			iconv('UTF-8','GB2312',"F:\\000_booktran\\combine5\\合并\\01-1\\术语替换\\返稿\\原文比较\\B.txt"),
		);
		foreach($fileNameArray as $fileName){
			$inputFile = file_get_contents($fileName);
			$outputFileName = str_replace(".txt","",$fileName)."-output.txt";
			file_put_contents($outputFileName,"");
			$inputLen = mb_strlen($inputFile);
			for($i=0;$i<$inputLen;$i++){
				$curChar = mb_substr($inputFile,$i,1);
				$curCharLen = strlen($curChar);
				echo "$curChar $curCharLen</br>";
				if ($curCharLen<>1)
					file_put_contents($outputFileName,$curChar,8);
				else
					continue;
				/*
				if (($curChar=="\r") || ($curChar=="\n")){
					file_put_contents($outputFileName,8);
					continue;
				}
				*/
			}
		}
	}	
    public function isEnglish($str)
    {
		$len=mb_strlen($str);
		if (($len%2)==0)
			$len -= 1;
		
		$enCount=0;
		for($i=0;$i<$len;$i++){
			$curChar = mb_substr($str,$i,1);
			if (strlen($curChar)==1)
				$enCount++;
		}
		
		return (2*$enCount>$len);
			
	}
	
	//calcuate direct contained in
	public function calcDirectContainedIn()	{
		set_time_limit(0);
		$termTableName = "term_78";
		//calc
		$terms = term::select("select id, chinese_key, d_container from $termTableName");
		foreach($terms as $term){
			$d_container = $term->d_container;
			
			$containing_terms = term::select("select id, chinese_key from $termTableName where chinese_key like \"%{$term->chinese_key}%\" and id<>{$term->id} order by length(chinese_key) desc");
			foreach($containing_terms as $k=>$containing_term){
				$foundIntermediate = false;
				for($i=$k+1;$i<count($containing_terms);$i++){
					if (mb_strpos($containing_term->chinese_key, $containing_terms[$i]->chinese_key)!==false){
						$foundIntermediate = true;
						break;
					}
				}
				if ($foundIntermediate==false){
					echo "term {$term->chinese_key} has a direct containing {$containing_term->chinese_key}</br>";
					
					$alreadyDContainer=false;
					$d_container_list = explode(",",$d_container);
					
					foreach($d_container_list as $d_container_item){
						if ($d_container_item==$containing_term->id){
							$alreadyDContainer=true;
							break;
						}
					}
					if ($alreadyDContainer==false){
						if (empty($d_container))
							$d_container = $containing_term->id;
						else
							$d_container .= ",{$containing_term->id}";
					}
					term::select("update $termTableName SET d_container=\"$d_container\" where id={$term->id}");
				}
			}
		}
		
		//print
		echo "******begin to print******</br>";
		$terms = term::select("select id, chinese_key, d_container from $termTableName");
		foreach($terms as $term){
			$d_container = $term->d_container;
			if (empty($d_container))
				continue;
			$d_container_list = explode(",",$d_container);
			foreach($d_container_list as $d_container_item){
				$temp_r = term::select("select chinese_key from $termTableName where id={$d_container_item}");
				if (empty($temp_r)){
					echo "error, some d_containing_item not found</br>";
				}else{
					echo "term {$term->chinese_key} has a direct containing {$temp_r[0]->chinese_key}</br>";
				}
			}
		}
		
	}
	
	
	//deduct # of appearance in containing terms 
	public function processFreqDirectContainedIn()	{
		set_time_limit(0);
		$termTableName = "term_78";
		
		$terms = term::select("select id, chinese_key, d_container,freq from $termTableName where source is null and category is null");
		foreach($terms as $term){
			$d_container = $term->d_container;
			$freq = $term->freq;
			
			if (empty($d_container))
				continue;
			$d_container_list = explode(",",$d_container);
			$max_d_container_freq = 0;
			foreach($d_container_list as $d_container_item){
				$temp_r = term::select("select chinese_key,freq from $termTableName where id={$d_container_item}");
				if (empty($temp_r)){
					echo "error, some d_containing_item not found</br>";
				}else{
					$max_d_container_freq = max($max_d_container_freq,$temp_r[0]->freq);
				}
			}
			if (($max_d_container_freq>0)&&(!empty($temp_r))){
				$freq -= $max_d_container_freq;
				echo "term {$term->chinese_key} has a direct containing {$temp_r[0]->chinese_key}. Freq decrease by {$temp_r[0]->freq}and is now $freq</br>";
			}
			//term::select("update $termTableName SET freq=$freq where id={$term->id}");
		}
		
	}
	
	public function fillTermSuper(){
		set_time_limit(0);
		$termTableName = "term_78";
		$terms = term::select("select id, chinese_key from $termTableName where source is not null or category is not null");
		foreach($terms as $term){
			$big_terms = term::select("select id, chinese_key from $termTableName where source is null and category is null and chinese_key like \"%{$term->chinese_key}%\" and id<>{$term->id}");
			foreach($big_terms as $big_term){
				echo "I am {$big_term->chinese_key}, bigger than {$term->chinese_key}</br>";
			}
		}
	}
	public function copyFromTerm(){
		set_time_limit(0);
		$fromTermTableName = "term";
		$toTermTableName = "term_78";
		$terms = term::select("select $toTermTableName.id, $fromTermTableName.source from $toTermTableName, $fromTermTableName where $toTermTableName.chinese_key= $fromTermTableName.chinese_key");
		foreach($terms as $term){
			term::select("update $toTermTableName SET source=\"{$term->source}\" where id={$term->id}");
		}
	}
	public function export78(){
		$temp_r=sentence::select("select sentence_78.content from sentence_78, article_78 where article_78.id=sentence_78.article_id and article_type=1 order by article_id, seq_no");
		foreach($temp_r as $sentence)
			echo "{$sentence->content}</br>";
	}
	
	public function cleanLong(){
		$termTableName = "term_78";
		$long = array(
"四大种所造","谓四大种所造眼识所依清净色","谓四大种所造耳识所依清净色","谓四大种所造鼻识所依清净色","谓四大种所造舌识所依清净色","谓四大种所造身识所依清净色","执受大种为因","乐依耽嗜受","乐依出离受","爱不相应受","无少所有无所有处","能断诸烦恼障","名为胜处净信","菩萨乘富伽罗","子虽小亦不可轻","大师在中说","尸罗圆满之所摄受","甚大殊异甚大高胜","当知一切出家菩萨","又复一切出家菩萨","在家菩萨则不如是","大众诸佛子","菩萨摩诃萨成就四法","灭先所造久积过罪",		
		/*
			"菩萨何用于声闻乘相应法教听闻受持",
			"外道书论精勤研究",
			"令一向习小法者舍彼欲故",
			"回向阿耨多罗三藐三菩提",
			"诸如来正觉所说法毗奈耶",
			"诸菩萨安住菩萨净戒律",
			"是为持戒生般若波罗蜜",
			"所起口业名为正语",
			"所起身业名为正业",
			"能令佛法久住",*/
		);
		/*$temp_r = term::select("select chinese_key from $termTableName where category=\"patriarchname\" and length(chinese_key)>9");
		foreach($temp_r as $v)
			$long []= $v->chinese_key;*/
			
		$sub_terms = term::select("select id, chinese_key from $termTableName where length(chinese_key)>=6 and source is null and category is null");
		foreach($sub_terms as $sub_term){
			$found=false;
			foreach($long as $long_item){
				if (mb_strpos($long_item, $sub_term->chinese_key)!==false){
					echo "$sub_term->chinese_key  $long_item</br>";
					term::select("delete from $termTableName where id={$sub_term->id}");
				}
			}
		}
		
	}
	
	public function importTermFromFile(){
		$articleFileName = iconv('UTF-8','GB2312',"F:\\000_booktran\\book7&8_term\\articles.txt");
		$articleTermsFileName = iconv('UTF-8','GB2312',"F:\\000_booktran\\book7&8_term\\articles-terms.txt");
		//$articleFileName = iconv('UTF-8','GB2312',"F:\\000_booktran\\book7&8_term\\test.txt");
		//$articleTermsFileName = iconv('UTF-8','GB2312',"F:\\000_booktran\\book7&8_term\\test-terms.txt");
		$articleFile = file_get_contents($articleFileName);
		$articleTermsFile = file_get_contents($articleTermsFileName);
		
		$articleLines = explode("\n",$articleFile);
		$articleTermsLines = explode("\n",$articleTermsFile);
		
		foreach($articleTermsLines as $k=>$articleTermLine){
			//echo "line=$articleTermLine</br>";
			$terms = explode(" ",$articleTermLine);
			foreach($terms as $term){
				if (empty($term)){
					echo "Error:term empty</br>";
				}
				//echo "$term </br>";
			}
			
			/*
			// workable, just to print
			echo sprintf("%04u:  ",$k+1);
			if ($terms[0]=="bb"){
				echo $articleLines[$k];
			}else{
				$decoratedLine = addDecoration($articleLines[$k],$terms,"<font color=\"red\">","</font>");
				if ($decoratedLine===false)
					echo "Error: pattern not matched in the line</br>";
				else
					echo "$decoratedLine</br>";
			}
			//end of  workable, just to print
			*/

			if ($terms[0]=="bb")
				continue;
			
			foreach($terms as $term){
				//lookup into term
				$temp_r=term::select("select id from term where chinese_key=\"$term\"");
				if (!empty($temp_r))
					continue;
				//$temp_r=term::select("select chinese_key, freq from term_78 where chinese_key=\"$term\"");
				//echo "$k+1|$term|".( (!empty($temp_r)) ? $temp_r[0]->freq : 0 )."|".addDecoration($articleLines[$k],$terms,"<font color=\"red\"> "," </font>")."</br>";
				echo "$k+1|$term|".substr_count($articleFile,$term)."|".addDecoration($articleLines[$k],$terms,"<font color=\"red\"> "," </font>")."</br>";
			}
			
		}
	}
	
	public function addRefNo(){
		$sentences = sentence::select("delete from sentence_78_ref where integrated=1");
		$sentences = sentence::select("select * from sentence_78_ref");
		$notExistPattern = "NotEXist";
		$curPattern = $notExistPattern;
		$curIntegratedSentence = "";
		$curArticleId = $curSentenceId = 0;
		foreach($sentences as $sentence){
			$pattern = "/A.*\..*/";
			$subject = trim($sentence->content);
			if (preg_match($pattern,$subject)){
				echo "{$sentence->content} new pattern matched</br>";
				if ($curPattern<>$notExistPattern){
//					echo "curPattern=$curPattern  curIntegratedSentence=$curIntegratedSentence</br>";
//					sentence::select("insert into sentence_78_ref (refno, article_id,  content,translation,integrated) values (\"$curPattern\", $curArticleId, \"".addslashes($curIntegratedSentence)."\",\"\", 1");
					sentence::select("insert into sentence_78_ref (refno, article_id,  content,translation,integrated) values (\"$curPattern\", $curArticleId, \"".addslashes($curIntegratedSentence)."\",\"\", 1)");
				}
				$curIntegratedSentence="";
				$curPattern = $subject;
				$curArticleId = $sentence->article_id;
				$curSentenceId = $sentence->id;
			}
			else{
				$curIntegratedSentence .= $sentence->content;
			}
		}
		echo "curPattern=$curPattern  curIntegratedSentence=$curIntegratedSentence</br>";
		//insert
	}
	
	public function matchSutra(){
		$option = "7goodsource";
		
		switch($option){
			case "7goodsource":
				$inputFileName = iconv('UTF-8','GB2312',"F:\\000_booktran\\book7&8引文\\《持戒与觉悟》引文查找-p3.txt");
				$outputFileName = iconv('UTF-8','GB2312',"F:\\000_booktran\\book7&8引文\\《持戒与觉悟》引文查找-p3-refno.txt");
				break;
			
			case "8zhang":
				$inputFileName = iconv('UTF-8','GB2312',"F:\\000_booktran\\book7&8引文\\第八册引文搜索(1-40)(胡仕翔)20180310-贤玲师整理-张纯翻译.txt");
				break;
				
			case "8all":
				$inputFileName = iconv('UTF-8','GB2312',"F:\\000_booktran\\book7&8引文\\第八册引文搜索(1-40)(胡仕翔)20180310-贤玲师整理.txt");
				$outputFileName = iconv('UTF-8','GB2312',"F:\\000_booktran\\book7&8引文\\第八册引文搜索(1-40)(胡仕翔)20180310-贤玲师整理-refno.txt");
				break;
		}
		$file = file_get_contents($inputFileName);
		$lines = explode("\r\n",$file);
		//var_dump($lines);
		$notFoundCount=0;
		$nonblankCount=0;
		$minMatchedSimilarity = 1;
		foreach($lines as $k=>$line){
			$fields = explode("\t",$line);
			echo sprintf("%04u:</br>",$k);
			foreach($fields as $k2=>$field){
				if($option=="8all"){
					if ($k2>0)
						continue;
				} else if ($option=="7goodsource"){
					if ($k2<3)
						continue;
				}

				if (empty($field)){
					//echo "ignore empty line</br>";
					if ($option=="8all"){
						file_put_contents($outputFileName,"Ignore empty field\r\n",8);
					}
					continue;
				}
				$nonblankCount++;
				
				//echo "field=$field</br>";
				
				if ($this::isEnglish($field)){
					//echo "ignore English line $field</br>";
					file_put_contents($outputFileName,"Ignore English field\r\n",8);					
					continue;
				}
				//echo "$k,$k2,$field</br>";
				
				$field=$this::stripBookName(trim($field));
				if (empty($field)){
					if ($option=="8all"){
						file_put_contents($outputFileName,"Ignore Book Name field\r\n",8);
					}
					continue;
				}
				
				$r=$this::matchSutraCore($field);
				if ($r['matched']===false){
					if ($option=="8all"){
						file_put_contents($outputFileName,"Not Found\r\n",8);
					}
					if($option=="7goodsource"){
						echo "$field not found</br>";
					}
					$notFoundCount++;
				}
				else{
					if ($option=="8all"){
						file_put_contents($outputFileName,"{$r['maxMatchRefNo']}|{$r['maxMatch']}\r\n",8);
					} else if($option=="7goodsource"){
						$english = ($k2<count($fields)-1) ? $fields[$k2+1] : "";
						file_put_contents($outputFileName,"{$r['maxMatchRefNo']}|{$r['maxMatch']}|$english\r\n",8);
					}
					if ($r['maxSimilarity']<$minMatchedSimilarity)
						$minMatchedSimilarity=$r['maxSimilarity'];
					
				}
			}
		}
		echo "notFoundCount=$notFoundCount nonblankCount=$nonblankCount</br>";
		echo "minMatchedSimilarity=$minMatchedSimilarity</br>";
	}

	public function matchSutraCore($str){
		$sources = sentence::select("select * from sentence_78_ref where integrated=1");// and id=12725");

		$maxSimilarity = 0.0001;
		$maxMatch = "";
		//echo "str before strip =$str</br>";
		//$str = $this::stripBookName(trim($str));
		//echo "str after strip =$str</br>";
		
		foreach($sources as $k=>$source){
			//$content=$this::addReturn($source->content);
			$content=$this::stripBookName(trim($source->content));
			if (empty($content))
				continue;
			//echo "str=$str</br>content=$content</br>";
			$similarity = $this::mb_similar($content,$str);
			if ($similarity>$maxSimilarity){
				$maxSimilarity = $similarity;
				$maxMatch = $content;
				$maxMatchRefNo = $source->refno;
			}
			/*if ($similarity>0.1)
				echo "candidate $k is $content similarity is $similarity</br>";*/
			/*
			$str=$this::addReturn($str);
			$content_len = mb_strlen($content);
			$str_len = mb_strlen($str);
			echo "after addReturn and mb_strlen</br>";
			$diffs = Diff::compare($content, $str,false);
			echo "after Diff::compare</br>";
			$sameCount = 0;
			foreach($diffs as $diff){
				if ($diff[1]==0)
					$sameCount++;
			}
			echo "$content_len  $str_len $sameCount</br>";*/
		}
		if ($maxSimilarity<0.5){
			echo "$str not found</br>";
			echo "maxMatch is $maxMatch maxSimilarity is $maxSimilarity</br>";
			return array("matched"=>false,"maxSimilarity"=>0, "maxMatch"=>"", "maxMatchRefNo"=>"");
		}
		else{
			//echo "str is $str</br>maxMatch is $maxMatch</br>";
			return array("matched"=>true,"maxSimilarity"=>$maxSimilarity, "maxMatch"=>$maxMatch, "maxMatchRefNo"=>$maxMatchRefNo);
		}
	}
	
	public function mb_similar($str1,$str2){
		//echo "str1=$str1<br>,str2=$str2</br>";
		$patternLen=2;
		$len1=mb_strlen($str1);
		$len2=mb_strlen($str2);
		$maxLen = max($len1,$len2);
		$minLen = min($len1,$len2);
		if ($maxLen>($minLen*2))
			return 0;
		$foundCount=0;
		for($i=0;$i<$len1-$patternLen+1;$i++){
			$subStr = mb_substr($str1,$i,$patternLen);
			if (mb_strpos($str2,$subStr)!==false)
				$foundCount++;
		}
		return $foundCount/($len1-$patternLen);
	}
	
	public function addReturn($str){
		$len = mb_strlen($str);
		$array = array();
		for($i=0;$i<$len;$i++){
			$array []= mb_substr($str,$i,1);
		}
		return implode("\r",$array);
	}
	
	public function stripBookName($str){
		//$patternArray = array("/《.*》说：/","/《.*》云：/","/（《.*》卷第.*）/",/*"/[0-9]+/","/（卷第.*）/",*/);
		$patternArray = array("/《.{1,30}》说：/","/《.{1,30}》云：/","/（《.{1,30}》卷第.*）/","/[0-9]+/","/（卷第.*）/");
		$ret = $str;
		foreach($patternArray as $pattern)
			$ret = preg_replace($pattern,"",$ret);
		//echo "str=$str</br>ret=$ret</br>";
		return $ret;
	}
	
	public function deleteTermFromFile(){
		$termInputFileName = iconv('UTF-8','GB2312',"F:\\000_booktran\\book7&8_term\\articles-terms-v32.txt");
		$termOutputFileName = iconv('UTF-8','GB2312',"F:\\000_booktran\\book7&8_term\\articles-terms-v33.txt");
		$deleteSpecFileName = iconv('UTF-8','GB2312',"F:\\000_booktran\\book7&8_term\\possibleDelete.txt");
		$deleteSpecFile = file_get_contents($deleteSpecFileName);
		$lines=explode("\n",str_replace("\r\n","\n",$deleteSpecFile));
		$deleteSpec = array();
		foreach($lines as $line){
			$fields = explode(" ",$line);
			foreach($fields as $field){
				if (empty($field))	continue;
				$deleteSpec []= $field;
			}
		}
		var_dump($deleteSpec);
		$termInputFile = file_get_contents($termInputFileName);
		$lines=explode("\n",str_replace("\r\n","\n",$termInputFile));
		foreach($lines as $line){
			$inputFields = explode(" ",$line);
			$outputFields = array();
			foreach($inputFields as $field){
				if (empty($field))	continue;
				if (!in_array($field,$deleteSpec)){
					$outputFields []= $field;
				}else{
					echo "$field should be deleted</br>";
				}
			}
			$outputField=implode(" ",$outputFields);
			file_put_contents($termOutputFileName,"$outputField\r",8);
		}
	}
	
	public function showTermSentence(){
		//$chinese_key = "菩提";	$english_key_list = array("enlightenment");
		//$chinese_key = "发心";	$english_key_list = array("the spirit of enlightenment","the development of enlightenment","enlightenment");
		//$chinese_key = "在家";	$english_key_list = array("householder","lay","household","layperson");
		$chinese_key = "戒律";	$english_key_list = array("ethical discipline");
		
		if (!empty($english_key_list)){
			$english_key_list2=array();
			foreach($english_key_list as $english_key){
				$english_key_list2 [$english_key] = mb_strlen($english_key);
			}
			var_dump($english_key_list2);
			arsort($english_key_list2);
			
			$english_key_list = array();
			foreach($english_key_list2 as $k=>$v){
				$english_key_list [$k] = 0;
			}
		}
		var_dump($english_key_list);
		$temp_r=term::select("select chinese_key,sentence.content,sentence.translation from term, term_detail,sentence where term.id=term_detail.term_id and term_detail.sentence_id=sentence.id and has_big_term=0 and term_detail.article_id=56 and chinese_key=\"$chinese_key\"");
		//$temp_r=term::select("select chinese_key,sentence.content,sentence.translation from term, term_detail,sentence where term.id=term_detail.term_id and term_detail.sentence_id=sentence.id and has_big_term=0 and term_detail.article_id=56 and chinese_key=\"$chinese_key\" and sentence.translation not like \"%clear%\" and sentence.translation not like \"%pure%\" and sentence.translation not like \"%purify%\" and sentence.translation not like \"%purified%\"");
		
		foreach($temp_r as $temp_r_v){
			$part = "NOT FOUND";
			foreach($english_key_list as $k=>$v){
				//echo "k=$k v=$v</br>";
				
				//if (mb_strpos($temp_r_v->translation,$k)!==false){
				if (!empty(pos_in_sentence($k,$temp_r_v->translation))){	
					$english_key_list[$k]++;
					$part=$k;
				}
			}
			echo "{$temp_r_v->content}</br>{$temp_r_v->translation}|$part</br></br>";
		}
		echo count($temp_r);
		var_dump($english_key_list);
	}

	public function searchHighFreqWords(){
		$puncturesEN = array(",","\.","\?","!",":",'\"',"\'","-",";","\(","\)","\[","\]");
		$puncturesCN = array("，","。","？","！","：",'“','”',"‘","’","-","；","《","》","、","（","）","…","·","—","　");
		$puncturesPattern="/[0-9]|".implode("|",array_merge($puncturesEN,$puncturesCN))."/";	
		
		$chinese_key = "清净";
		
		$temp_r=term::select("select chinese_key,sentence.content,sentence.translation from term, term_detail,sentence where term.id=term_detail.term_id and term_detail.sentence_id=sentence.id and has_big_term=0 and term_detail.article_id=56 and chinese_key=\"$chinese_key\"");
		$allWords=array();
		foreach($temp_r as $temp_r_v){
			$sentence = $temp_r_v->translation;
			$sentence = preg_replace($puncturesPattern," ",$sentence);	
			//echo "$sentence</br>";
			$allWords = array_merge($allWords,explode(" ",$sentence));
		}
		//print_r(array_count_values($allWords));
		$b=array_count_values($allWords);
		arsort($b);
		print_r($b);
		
	}
} //!- NLPController
?>
